In [4]:
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import time
import datetime as dt
import os
from tqdm import tqdm
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
from dtw import dtw
from sklearn.metrics import pairwise_distances
PRE_NAME = "onemin_ohlc_"
BEGIN_TIME = "09:00:00"
END_TIME = "11:00:00"
NUM_CLUSTER = 50
def load_data():
X = []
Y = []
# df = pd.read_csv(os.path.join('dataset', '2327', PRE_NAME+"20180612.csv"))
# mask = (df.loc[:, "time"] >= BEGIN_TIME) & (df.loc[:, "time"] <= END_TIME)
# front_df = df[mask].loc[:, "return"]
# end_df = df[~mask].loc[:, "return"]
# X.append(np.array(front_df))
# Y.append(np.array(end_df))
# """我是分隔線^^~"""
# df = pd.read_csv(os.path.join('dataset', '2327', PRE_NAME+"20180613.csv"))
# mask = (df.loc[:, "time"] >= BEGIN_TIME) & (df.loc[:, "time"] <= END_TIME)
# front_df = df[mask].loc[:, "return"]
# end_df = df[~mask].loc[:, "return"]
# X.append(np.array(front_df))
# Y.append(np.array(end_df))
for sid in tqdm(os.listdir(os.path.join('dataset'))):
for file in os.listdir(os.path.join('dataset', sid)):
# print(file)
df = pd.read_csv(os.path.join('dataset', sid, file))
df = df.take(np.arange(0, len(df), 5))
mask = (df.loc[:, "time"] >= BEGIN_TIME) & (df.loc[:, "time"] <= END_TIME)
front_df = df[mask].loc[:, "return"]
# print(front_df)
# exit()
end_df = df[~mask].loc[:, "return"]
if len(front_df) == (121 // 5 +1):
X.append(np.array(front_df))
Y.append(np.array(end_df))
# print(end_df)
# df = pd.read_csv(os.path.join('dataset', '2327', PRE_NAME+"20180612.csv"))
# print(len(X))
# print(len(X[0]))
# X = np.array(X)
# print(X.shape)
# exit()
return np.array(X), np.array(Y)
def dtw_d(X, Y):
manhattan_distance = lambda x, y: np.abs(x - y)
d, cost_matrix, acc_cost_matrix, path = dtw(X, Y, dist=manhattan_distance)
return d
def dtw_affinity(X):
return pairwise_distances(X, metric=dtw_d)
X, Y = load_data()
In [5]:
print(X.shape)
In [6]:
ac = AgglomerativeClustering(n_clusters = NUM_CLUSTER,
affinity = dtw_affinity,
linkage = 'complete')
X_label = ac.fit_predict(X)
In [7]:
X_label
Out[7]:
In [8]:
for n in range(NUM_CLUSTER):
for i in range(len(X)):
# plt.subplot(10, 1, X_label[i]+1)
if X_label[i] == n:
plt.plot(X[i])
plt.show()
In [9]:
def make_long_simple(y, cost):
return y.max() - y[0] - cost
def make_short_simple(y, cost):
return y[0] - y.min() - cost
def make_long_max_lost(y, cost):
return y.min() - y[0] - cost
def make_short_max_lost(y, cost):
return y[0] - y.max() - cost
In [10]:
profit_long_array = np.zeros(NUM_CLUSTER)
profit_short_array = np.zeros(NUM_CLUSTER)
lost_long_array = np.zeros(NUM_CLUSTER)
lost_short_array = np.zeros(NUM_CLUSTER)
num_long_array = np.zeros(NUM_CLUSTER).astype("int")
num_short_array = np.zeros(NUM_CLUSTER).astype("int")
In [ ]:
In [11]:
profit_long_array.shape
Out[11]:
In [12]:
for i in range(len(X)):
profit_long_array[X_label[i]] += make_long_simple(Y[i], 0.002)
profit_short_array[X_label[i]] += make_short_simple(Y[i], 0.002)
lost_long_array[X_label[i]] += make_long_max_lost(Y[i], 0.002)
lost_short_array[X_label[i]] += make_short_max_lost(Y[i], 0.002)
num_long_array[X_label[i]] += 1
num_short_array[X_label[i]] += 1
print(profit_long_array)
print(profit_short_array)
print(lost_long_array)
print(lost_short_array)
print(num_long_array)
print(num_short_array)
In [13]:
import pickle
pickle.dump(ac, open("ac_model2.pc", 'wb'))
In [ ]: